import scrapy
import json
from scrapy import Request
from contentSpider.items import ContentItem
from scrapy import log
from contentSpider.settings import UID

class contentSpider(scrapy.Spider):
    name = "contentSpider"

    #用户主要信息 不需要登入
    user_url = 'https://m.weibo.cn/api/container/getIndex?type=uid&value={uid}'
    #用户所有的动态 不需要登入
    content_url = 'https://m.weibo.cn/api/container/getIndex?type=uid&containerid={containerid}&page={page}'

    def start_requests(self):
        yield Request(self.user_url.format(uid=UID), callback=self.parse_user,meta={'uid':UID},dont_filter=True)


    def parse_user(self, response):
        """
        解析用户信息 用于获取containerid
        :param response: Response对象
        """
        resData = json.loads(response.body).get('data')
        for data in resData.get('tabsInfo').get('tabs'):
            if(data.get('tab_type') == 'weibo'):
                containerid = data.get('containerid')
                uid = response.meta.get('uid')
                #这里爬取所有的用户动态
                yield Request(self.content_url.format(containerid=containerid, page=1), callback=self.parse_content, meta={'page': 1, 'containerid': containerid,'uid':uid},dont_filter=True)


    #获取所有的动态
    def parse_content(self, response):
        #这里处理动态数据
        #需要登入,直接返回
        text = json.loads(response.body).get('data')
        cards = text.get('cards')
        if (len(cards) > 0):
            page = response.meta.get('page')
            containerid = response.meta.get('containerid')
            uid = response.meta.get('uid')
            log.msg("-----正在爬取第" + str(page) + "页-----", level=log.INFO)
            contentIds=[]
            for j in range(len(cards)):
                card_type = cards[j].get('card_type')
                if (card_type == 9):
                    #微博动态id
                    mblog = cards[j].get('mblog')
                    contentId = mblog.get('id')
                    contentIds.append(contentId)
            contentItem = ContentItem()
            contentItem['uid']=uid
            contentItem['contentIds'] = contentIds
            yield contentItem
            page+=1
            yield Request(self.content_url.format(containerid=containerid, page=page),
                          callback=self.parse_content, meta={'page': page, 'containerid': containerid,'uid':uid},dont_filter=True)
        else:
            return